{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import init, gluon, nd, autograd, image\n",
    "from mxnet.gluon import nn\n",
    "import numpy as np\n",
    "import pickle as p\n",
    "import matplotlib.pyplot as plt\n",
    "from time import time\n",
    "%matplotlib inline\n",
    "ctx = mx.gpu()\n",
    "data_dir = '/home/sinyer/python/data/cifar10'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_cifar(route = data_dir+'/cifar-10-batches-py'):\n",
    "    def load_batch(filename):\n",
    "        with open(filename, 'rb')as f:\n",
    "            data_dict = p.load(f, encoding='latin1')\n",
    "            X = data_dict['data']\n",
    "            Y = data_dict['labels']\n",
    "            X = X.reshape(10000, 3, 32,32).astype(\"float\")\n",
    "            Y = np.array(Y)\n",
    "            return X, Y\n",
    "    def load_labels(filename):\n",
    "        with open(filename, 'rb') as f:\n",
    "            label_names = p.load(f, encoding='latin1')\n",
    "            names = label_names['label_names']\n",
    "            return names\n",
    "    label_names = load_labels(route + \"/batches.meta\")\n",
    "    x1, y1 = load_batch(route + \"/data_batch_1\")\n",
    "    x2, y2 = load_batch(route + \"/data_batch_2\")\n",
    "    x3, y3 = load_batch(route + \"/data_batch_3\")\n",
    "    x4, y4 = load_batch(route + \"/data_batch_4\")\n",
    "    x5, y5 = load_batch(route + \"/data_batch_5\")\n",
    "    test_pic, test_label = load_batch(route + \"/test_batch\")\n",
    "    train_pic = np.concatenate((x1, x2, x3, x4, x5))\n",
    "    train_label = np.concatenate((y1, y2, y3, y4, y5))\n",
    "    return train_pic, train_label, test_pic, test_label\n",
    "\n",
    "def accuracy(output, label):\n",
    "    return nd.mean(output.argmax(axis=1)==label).asscalar()\n",
    "\n",
    "def evaluate_accuracy(test_data, net, ctx):\n",
    "    acc = 0.\n",
    "    for data, label in test_data:\n",
    "        data = data.as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        output = net(data)\n",
    "        acc = acc + accuracy(output, label)\n",
    "    return acc / len(test_data)\n",
    "\n",
    "def augment(data, auglist):\n",
    "    data = nd.pad(data, pad_width=(0,0,0,0,2,2,2,2),mode='constant',constant_value=0)\n",
    "    data = nd.transpose(data, (0,2,3,1))\n",
    "    temp = []\n",
    "    for d in data:\n",
    "        for aug in auglist:\n",
    "            d = aug(d)\n",
    "        temp.append(d)\n",
    "    data = nd.stack(*temp)\n",
    "    data = nd.transpose(data, (0,3,1,2))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Residual(nn.Block):\n",
    "    def __init__(self, channels, same_shape=True, equal=True, **kwargs):\n",
    "        super(Residual, self).__init__(**kwargs)\n",
    "        self.same_shape = same_shape\n",
    "        self.equal = equal\n",
    "        with self.name_scope():\n",
    "            strides = 1 if same_shape else 2\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.conv1 = nn.Conv2D(channels, kernel_size=3, padding=1, strides=strides)\n",
    "            self.bn2 = nn.BatchNorm()\n",
    "            self.conv2 = nn.Conv2D(channels, kernel_size=3, padding=1)\n",
    "            if (not same_shape) or (not equal):\n",
    "                self.conv3 = nn.Conv2D(channels, kernel_size=1, strides=strides)\n",
    "    def forward(self, x):\n",
    "        out = self.conv1(nd.relu(self.bn1(x)))\n",
    "        out = self.conv2(nd.relu(self.bn2(out)))\n",
    "        if (not self.same_shape) or (not self.equal):\n",
    "            x = self.conv3(x)\n",
    "        return out + x\n",
    "\n",
    "class wrn(nn.Block):\n",
    "    def __init__(self, num_classes, **kwargs):\n",
    "        super(wrn, self).__init__(**kwargs)\n",
    "        with self.name_scope(): \n",
    "            net = self.net = nn.Sequential()\n",
    "            net.add(nn.Conv2D(channels=16, kernel_size=3, strides=1, padding=1))\n",
    "            net.add(Residual(channels=16*8, equal=False))\n",
    "            net.add(Residual(channels=16*8), Residual(channels=16*8))            \n",
    "            net.add(Residual(channels=32*8, same_shape=False))\n",
    "            net.add(Residual(channels=32*8), Residual(channels=32*8))\n",
    "            net.add(Residual(channels=64*8, same_shape=False))\n",
    "            net.add(Residual(channels=64*8), Residual(channels=64*8))\n",
    "            net.add(nn.BatchNorm())\n",
    "            net.add(nn.Activation(activation='relu'))\n",
    "            net.add(nn.AvgPool2D(pool_size=8))\n",
    "            net.add(nn.Flatten())\n",
    "            net.add(nn.Dense(num_classes))\n",
    "    def forward(self, x):\n",
    "        out = x\n",
    "        for i, b in enumerate(self.net):\n",
    "            out = b(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_pic, train_label, test_pic, test_label = load_cifar()\n",
    "\n",
    "batch_size = 128\n",
    "train_pic = np.transpose(train_pic.astype('float32')/255, (0,2,3,1))\n",
    "test_pic = np.transpose(test_pic.astype('float32')/255, (0,2,3,1))\n",
    "mean = [0.4914, 0.4822, 0.4465]\n",
    "std = [0.2470, 0.2435, 0.2616]\n",
    "for i in range(3):\n",
    "    train_pic[:,:,:,i] = (train_pic[:,:,:,i] - mean[i])/std[i]\n",
    "    test_pic[:,:,:,i] = (test_pic[:,:,:,i] - mean[i])/std[i]\n",
    "train_pic = np.transpose(train_pic, (0,3,1,2))\n",
    "test_pic = np.transpose(test_pic, (0,3,1,2))\n",
    "train_data = gluon.data.DataLoader(gluon.data.ArrayDataset(train_pic, train_label.astype('float32')), \n",
    "                                   batch_size, shuffle=True)\n",
    "test_data = gluon.data.DataLoader(gluon.data.ArrayDataset(test_pic, test_label.astype('float32')), \n",
    "                                  batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_train = image.CreateAugmenter(data_shape=(3, 32, 32), rand_crop=True, rand_mirror=True)\n",
    "\n",
    "net = wrn(10)\n",
    "net.initialize(ctx=ctx, init=init.Xavier())\n",
    "loss = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "trainer = gluon.Trainer(net.collect_params(), 'nag', {'learning_rate': 0.1, 'momentum': 0.9, 'wd': 5e-4})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 loss:1.4546 tracc:0.4655 teacc:0.4839 time:125.999\n",
      "10 loss:0.4043 tracc:0.8603 teacc:0.7781 time:126.190\n",
      "20 loss:0.3309 tracc:0.8867 teacc:0.8370 time:126.081\n",
      "30 loss:0.2994 tracc:0.8967 teacc:0.8195 time:126.381\n",
      "40 loss:0.2853 tracc:0.9018 teacc:0.7975 time:126.242\n",
      "50 loss:0.2816 tracc:0.9039 teacc:0.8415 time:126.346\n",
      "60 loss:0.1360 tracc:0.9563 teacc:0.9397 time:126.314\n",
      "70 loss:0.0405 tracc:0.9874 teacc:0.9302 time:126.328\n",
      "80 loss:0.0578 tracc:0.9819 teacc:0.9215 time:126.897\n",
      "90 loss:0.0600 tracc:0.9806 teacc:0.9178 time:126.217\n",
      "100 loss:0.0499 tracc:0.9837 teacc:0.9241 time:126.278\n",
      "110 loss:0.0581 tracc:0.9816 teacc:0.9054 time:126.234\n",
      "120 loss:0.0189 tracc:0.9951 teacc:0.9517 time:126.002\n",
      "130 loss:0.0023 tracc:0.9999 teacc:0.9552 time:125.890\n",
      "140 loss:0.0019 tracc:1.0000 teacc:0.9572 time:126.002\n",
      "150 loss:0.0019 tracc:1.0000 teacc:0.9568 time:125.694\n",
      "tracc:0.999980 teacc:0.956784\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXmYXFWZ/z9vVe9LOp3uztbZQxYS\nAgRCIARkEwiIoLiBGzoqbjiK48YwPxd0BHdxZMaJiijIJqIGzIjIKltIAtkXyJ5OJ+l9X2o7vz/O\nvV23K1Xd1Z3urkr1+3meem7drerUTfp7v/c973mPGGNQFEVRRge+VDdAURRFGTlU9BVFUUYRKvqK\noiijCBV9RVGUUYSKvqIoyihCRV9RFGUU0a/oi8jdIlIjIlsS7BcR+ZmI7BKRTSJyhmffDSLypvO6\nYSgbriiKogycZJz+PcCKPvZfAcxxXjcC/wMgIuOAbwBnA0uBb4hI6fE0VlEURTk++hV9Y8zzQEMf\nh1wD/M5YXgHGisgk4HLgSWNMgzGmEXiSvm8eiqIoyjCTNQSfUQkc9KxXOdsSbT8GEbkR+5RAYWHh\nmfPnzx+CZimKki40dwY53NxFMBxJdVPSmvxsPyeNLxrUuevXr68zxlT0d9xQiL7E2Wb62H7sRmNW\nAisBlixZYtatWzcEzVIUJR147o1aPnbPWi6aNIZLTh7PpJI8CnKyyM/2U5DjJzvLh08En+AsBZ/P\n817A55Nex0g8dckAcvw+xo/JG9S5IrI/meOGQvSrgKme9SlAtbP9wpjtzw7B9ymKcoLw5tFWPn3f\neuZMKOb3nzibMXnZqW7SqGcoUjZXAR92snjOAZqNMYeBJ4DLRKTU6cC9zNmmKMoo4eF1BwmFDb/9\n6Fkq+GlCv05fRB7AOvZyEanCZuRkAxhjfgGsBq4EdgEdwEedfQ0i8m1grfNRtxlj+uoQVhQlw3h6\nRw1nzxo36JCFMvT0K/rGmOv72W+AzybYdzdw9+CapijJE4kY/vhaFfvrOygtzGF2RSFnTC9Vd5lC\nDtR3sLu2nQ+cPT3VTVE8DEVMX1FGHGMMrx9sYldNG2PysrjnpX28sqcBEXCniBCBueOLWTKjlI8u\nnznorAhlcDy94ygAF88fn+KWKF5U9JUTiq5gmMc2VvPbl/ex5VBLz/bi3Cy+/+5TefcZU2juDLL9\ncAvr9jeybn8jf3r9EI+sr+KrK+Zz4bwKygpzKSnQJ4Dh5umdtcwqL2RGeWGqm6J4UNFXThie2HqE\nWx7dTEN7gDnji/j2O07hLXPKaekMUVmaz7jCHABKC3M496Ryzj2pHICali6++sdN3Pb4Nm573H7W\n/InFXDx/PJ+8YDYl+dEbQHt3iM5gmPKi3BH/fYOlKxjmqe01dAXDvO3USeRl+/s9pzMQ5hfP7eYv\nGw5x9WmT+eQFsynMPVYOjDG8vLuemtZuls4cR01rN39+/RA1rV34fT5Om1LCNadXUlHc+3q1d4d4\nZXc9H1qmoZ10Q9JtukTN01fisX5/A9f/cg3zJhRzyxXzWTa7DBlAsrYxhlf2NHCkpZPqpi5e3FXH\nmr0NTByTxzfevoCyohxe2dPAyuf30NYd4rqzpvKvl8xhgqcD0hjDo68d4khLF+VFOVw0b3yvDspA\nKMLmQ02cVFHc8yRhjOHFXfXUt3fztkWTyPL7iEQMYWPI9vdOnusKhsnyCVn+xEl1rx1o5JH1VQRD\nEbpDERraA2w+1ExzZxCAssIcPnjOdD60bDp+EVZvOczzb9Sybl8jk8bmsWxWGXVtAV7aXcfRlm4W\nVZaw+VAzxXlZTBiTR2lBNqdOGcv0sgLq2gI8tf0oW6tberUhN8vH1HEFdAXDVDV24vcJJ08q5pTJ\nJSydOY5p4wr44d938sqeBh74xDksm12W9L+TMnhEZL0xZkm/x6noK+nOwYYOrv75C5TkZ/PoZ5b3\nOPrj5fUDjdz80Ab21Xf0bLtk/ngmjc3jwVcPEooYTqkcw9sWTeaD50zjR39/g3te2tdzbH62n48s\nn8G8CcUcbenity/to7q5CxGYVV5IWVEuta3d7K1rB2BRZQkXzavgkfVVHG7poqIol7NnlfGpC2bx\n8u56fvT3NxCBUypLaO4IsreundxsHxXFuZxaWUK238cjr1VRmJPFmLwscrJ8lBbmMLO8kGsXT8Hv\nE379wh7+sb2GnCwfxhiCYcOU0nyWzhjHwcYOXjvQRFlhDqdNHcvHz5vJ2bPKWL+/kT+sO0hLV5Cj\nLd1sPtRMIBRBBOaML+Jj581k4eQS1uxtoCjXzxWLJvV0kL95tJW/bKhmw8GmXjefkvxsvrJiHu9f\nOm1AN2dl8KjoKxnDx3+7llf2NPDY585j5hDHhzsCIdbsacDnEyaV5DF3QjEA++ra+evmwzyzo4Z1\n+xvJyfIRCEX4+Hkz+dLl89hX385dz+zmsY3VPZ+1ZHopHzxnOgcbOnoEMNvv49ozKsny+7jtsa3U\ntQU4f045i6eOpaqpkye2HKE9EAZsh+fU0nw2VjUzzslACoYNh5s7ee1AE3Vt3dywbAZfunweRXFC\nMS67atq475X9ZPmEd55RyYJJY3qENxiOHPOEEUsgFKGxI0BZYU6fTx2xRCKGbYdb2H64hUtOnjBk\nN2clOVT0lYzgxV11fOBXa/jqivl8+sLZKWnDxoNNrHx+Dwsmj+EzF87u5VyPNHfRHgiRl+1nckle\nn662vTtEW3eoV8ioqSPAH9ZVMXVcAZcvnJDwfGMM3aFIUvF6ZXSioq+c8IQjhrf97J+0dYf4xxcv\nUMFTlD5IVvR15iwlbXl2Zw07jrTylRXzVfAVZYhQ0VfSltcPNOH3CZctmJDqpihKxqCir6Qtmw81\nM3dCsbp8RRlCVPSVtMQYw+ZDzSyqHJPqpihKRqGir6Ql1c1dNLQHWFRZkuqmKEpGoaKvpCWbq5oA\nWDRlbIpboiiZhYq+kpZsPtRMlk+YP7E41U1RlIxCRV9JSzYfamGOduIqypCjoq+kHcYYNlc1carG\n85XhIhSA1qMQtrWCiITterDTroeDEOiITs4wGCIRaNxvvyuN0NLKStpxqKmTxo4gp0xR0VdiiETA\n53jVo1thyx9h7DTILYZDr0FzlX0/djrMvhgE2P20FfPiSXBkM7z5d2g9bD9D/FA0AdprIeLcAPw5\nEHaEOisPxlTClLMgpxD2vQChLrtuwlC1DgJtkFNk97svXxZUb4DOBvt542ZDuNveAApKISvftqG7\nBXKKwZ9tv3PCKfCBh4f1EiYl+iKyArgT8AO/MsbcEbN/OnZaxAqgAfigMabK2RcGNjuHHjDGXD1E\nbVcylANO1cuTKnSmq4yju9WK9ZHNcGSTXTYfsuJXNAFmnA+VZ8C4WbD3edj6KGQXwJjJ0LDHOue5\nl8OEhfDinVFxBivQJVOtCLcegWe+4+wQEJ8V6ZxiOOkSK675Y6Gtxt4oisZbcQ+0QlezI8RZ0NFg\nv3f30xDsgGnLIKcADrxsP3PKWVBYDoF2+72BdvvqaoF5V9jf0nQA6nZBdp69AXQ22pvQjPMgd4w9\nLxwAfy6UzRr2f4JkJkb3A3cBlwJVwFoRWWWM2eY57IfA74wxvxWRi4HbgQ85+zqNMacPcbuVDKbJ\nKc9bWqizW53wGGMFcsujsPspK6Au+aUw8VSYd6p15437YOdfYcN9dr/4rFv350LzQSv0sy+BrX+C\nnath/lVw1U8h2A6dTTB+AWQ5lT3b62HPM/b9rAshb6x18wXjIGsQE+R45+A8wUnG6S8Fdhlj9gCI\nyIPANYBX9BcANzvvnwH+PJSNVEYXTR1W9Mfma2neE57tj8HDH7LhjNkXwWnvh4mL7GvM5GNF1Bjr\nvOvfhPJ5UFJ57Gde/p/QsBcq5jnnV0BpzDGFZbDo3b23jZk0+N+RAWLvkozoVwIHPetVwNkxx2wE\n3oUNAb0TKBaRMmNMPZAnIuuAEHCHMUZvCEqfNHXaR/axOo/tiU/Dbru8easV4v4QgbFT7SsRWbkw\nfv7QtG8Ukkz2TrxbXGyX9peAC0TkdeAC4BBW5AGmOeU+3w/8VESOKYouIjeKyDoRWVdbW5t865WM\npLkjSG6WT9M1M4G2GtvJmYzgKyNCMqJfBXhvu1OAau8BxphqY8y1xpjFwK3OtmZ3n7PcAzwLLI79\nAmPMSmPMEmPMkoqKisH8DiWDaOoIqsvPFNpqoFD/ptOJZMI7a4E5IjIT6+Cvw7r2HkSkHGgwxkSA\nW7CZPIhIKdBhjOl2jlkOfH8I269kIE2dgd7x/M4meO13NuvDROCiW2x2h5L+tB21WTlK2tCv6Btj\nQiJyE/AENmXzbmPMVhG5DVhnjFkFXAjcLiIGeB74rHP6ycD/ikgE+1RxR0zWj6IcQ1NHkBLX6YcC\n8OAHYP8LUDzZpvxtfwzO/zc448NQrIKS1rTV2A5XJW1IKk/fGLMaWB2z7eue948Aj8Q57yVg0XG2\nURlldHR08El5FF7fDftftoJ/7S/h1PdCSzX89Us2B/vZ70LZHDuopnQGLP6gTeMbTEqeMjy0HYWZ\nb0l1KxQPOiJXSTsqO7ZxVfA++IuTr33eF63gg03zu/5+qH0DNj0EdTvtgJeDa+GRf7EjH6/+mR34\n4iXUDa/8N5z5UTsoRxl+Qt3Q1WQHPilpg4q+knaEutptIPGau+ygmnlXHntQxVy45P9F1yNhO7z+\nb1+De95mR3bOvADO+pgdkLPh9/CPb0J2IZx940j9lNFNu5OJp6KfVmjBNSWt6AqG8Ye77MrEU+Hk\nq6K1VvrC57fD3j/9ErzlK7bz95nvwB8+AuEQvPRf9riqV4et7UoMbUftUjty0woVfSWtaOkMkotT\nTyU7f+AfkFMIF98Kn34B3vZj2Psc/OEGO/y/sAIOrhnaBiuJaauxS3X6aYWKvpJWNHUGyRNPhcPj\n4cyPwvTlsONxm+K5/PO2+FXL4eNvqEvjPlv5UTkW1+kXquinEyr6SlrR1BEk73icvhefD67+Lyia\nCBf+O0w7124fKrd/YA3ceTo89c2h+bxI+Pjqt6cb6vTTEhV9Ja1o6ghERf94nT5A2Wz44nY49T22\nyFdWHhx81cb5qzcMXmSNgae+BRh46edwZEt0X8MeeOwLNnslWSJhuPO0aN+Dl4F8TjrRVmM74jWF\nNq1Q0VfSiqbOIHk4k1kcr9N3cTuCs3Jg8hlw8BVYdROsvMAO/Dr0Gjx+M/z+vXA0ybGDu5+G/S/C\nRbfaEsGPfT4a5nnp57D+NwN7oqjZZssHb4upR3hoPXxvBrz++77PDwVsHfdEtFTbGx3YWaHWrLQD\n3YYTHY2blqjoK2lFc4eN6Rtfts3IGWqmLrVCuvEBmHsF7HoSfnkRvH6fzexZeSGs+03fn9GwB578\nup2xafkXbKnfQ+vsuIFQwE78AQMTfffYQ6/ZiTtc1v3GTt7x+M12XyIevxnuXnHs9pbD8KdPw48X\nwJNOiuuG++H/vmxnnfJS+wY894OhCzG11WhoJw1R0VfSiubOIPkSGDqXH8t0J65/5kfg+gfgxmfh\n8tvh8xvhs2th+jL467/ZDl+Atb+GjQ9Fz3/iVvivM6HuDbj8u/bp4dT3wYRF8PwP7FiBzkbwZduY\nf7IcfNVO3YexGUdg52jd+mc7TqFoPDz0ofjuPBKxk4oc2QT1u6Pbg13wq0tgyyN2gpG1v7Idzy/+\n1O73hqS6WuCB62yaa92bybe7L9qOquinISr6SlrR1BlgTFYYGYp4fjxOuhQ+9Ce48ke2dvuEhbDs\nM3akb1EFXP1zu/2V/4Ga7bD6y/D0t637bT0CL/8cFlwDX9gMJ7/dfqYIXPAVWzv+r1+EgjJY9B77\n5JBsZs/BNXYawNwSGzoC2PFXO33fOZ+Gt/8UWqpg/0vHnnt0i52LFeDNJ6Pbt/wRWg7BdQ/A+50b\n133vsk8q2QV2qkKwv23V56K17w9vGNg1jcV9Umir0fBOGqKir6QVTR1Biv1BO5/ocOBzp+BLMBh9\n7FQ45d2w/rc2ZGLCNtbesMfO2Qpw3s1QPLH3efOvgvELrbtd+E5bBqKr2ZaJcOlstE7b5dVf2lBL\n61G7ffq5MOstsPsZK5wb74eSaTD9PDs3q/hsaCoW98mgaAK8+YR9b4wtOzF+oZ0TduxUm8JavwvK\n58Jp19ubRSQCO//P9iVc/B+2o/vwxsFcWcvuZ+AHs+3vCrar009DVPSVtKK5M0iRP2Sn10sVy//V\nCtaBl+GsT9hte561r/xxNpQTi88HF/27FebT3w/TzrHb3Vh9KAC/uwbuucoKsjHw7O22A9idE3bq\nOfaG1HzQOu89z8Jp77OfnVMIFfPjx/X3PGcLzy16D+x7wXbo7n3eivo5n45O9Xf+F+3E4RfdCpNO\nsxNyN+2z89LmlcDym+2E4dWDdPoNe+wI6I56235Qp5+GqOgraUVTR5BC3zA6/WSYsDDq3FfcboVy\nzzNWhGddkLgsxMlXwZd3Q+WZdjBYQXk0rv/cHdZBNx+0c8A2HbDiGA7AU9+2k39POtWKPlinvPCd\nsOyz0c+vPAOqX+vd0RoK2JDPrAtgzmX28zY9ZENSBeX2RuBSPBFu3gIL32HTVwEOb4JdT9nJw/1Z\nMPl0285IxPZl/OEjNqwFtlPY28nspWYHPHC9ff+R1TZVE3RgVhqiBdeUtKKpM0C+BFPr9AHec4/N\nnfdnW0Hd9LAV1FkX9n1ewTi7FIGpZ9vQyzPfhRd+AlOW2jh/1dro8Wd+BNbfA5MX23z20hlww+NQ\nOt1mB3mZfIbNMmrab48DG+4JttvictOWQU6xDUv5c+Htdya+eY4/2XYcb/4DtB6Gk95qt0863Xb4\n1u+yN47mg7DvRfvksuNx286PPwWREPz6UhvCKp1pb4g5RfC+38GM5fb6rf4yTDwl6UuujAzq9JW0\nwDjutakjSL50D1/2TrL4s6OCOesiK/jQv+h7mXWh7Uh97ns2nPL+h2zMvGqtFWt/LlzxfRtfP/OG\n6Hkzzz9W8ME6fegd4tn9NCC2DyErB87+pH1K+czLcPr1iduWnQ/lc6yQA8y+xC4nnWaXL/7UCv5F\nt9ob2Z5n7Y3l0Hr7ZLH5D1D9un0Kaj4IZ30c/vX16PWZeT589pVj+z6UlKNOX+mXzkCYvGwf4saG\nk6C6qZMth5q5dMGEfs/bV9fOB361hredOonWrhB5hcOYsjkY3ElAxk6POuxkOOvjNiOnsNzG5ME6\n5aq1dg6ASadad//OXyT3eeMX2vOqX4NTrrUpnet/Y0NC7hOGt9x0f0xcBLU7oOJkKKl0vuNkezPa\n8HubhbT8C/CWL9unnnAAfrLQZjDV77J9Gx/+S7TPQDkhUKev9ElzR5ALfvAMn/jdOoLh5NIPjTH8\n6wOvc+O96/nOX7cTiRgON3fy+oFGthxqZueRVvbUttEVDNMRCPGp+9ZT09rFyuf3AJBjAkNTgmGo\nKBoPc1dYRz4QfD4bpnEFH2DKWTZmXr3BhmsGQlaOFWrX6a+/x9asv+ArA/scFzeuf9Il0W3+bNun\nAfb3ZuVYUfdnQU6BnZ9g52o7TuG8L6jgn4Co01d6aOoI8OzOWtbvb2RRZQnvPWsqdz71JrVt3fxj\new3//uhmbr92EREDh5o6qW7qpDgvi9KCHMIRQ162n4kleTy1vYZ1+xs5bepYfv3CXlZtrKa29dj6\nMfnZfiaNzWNvXTv3fHQpWw4184MndlrRTyenD9E89+Nlylnw0s+sa648c+DnV55pSzJUrYMX77ST\nxbiZQgNuy1K7jJ2kZtJp9mnijA8fe85Zn7DfWzLFdjQrJxxJib6IrADuxM5n9CtjzB0x+6cDdwMV\nQAPwQWNMlbPvBuA/nEO/Y4z57RC1XUmSw82d3Pvyfj75ltmUFGSzt66d+9fs5/w5FSybXUa238dr\nBxr51L3rqWntJsfv495X9rPtcAv3vbKf686aSkVxHj976k3+sL6qz++6Ydl0XtnTwMzyQh751DLu\ne2U/L++uZ+nMccyuKCIYjhCKGLqCYTYcbOKFN+u49cqTuWBuBRfMreAdiyvJW5lmTn8omXJW9P1g\nRP/kq+0o4V857vzalYNvy/Rl8PlN9mnEy7mfs+Uq4k1oXjwB3vUrO0n9cJTJUIadfkVfRPzAXcCl\nQBWwVkRWGWO8lal+CPzOGPNbEbkYuB34kIiMA74BLAEMsN45t3Gof4hiae4I4vcLRbn2n/ZgQwfv\n/9UrHGzopK07xLeuXshX/7iJV/c28Mt/7iU/28+sikLePNrGxJI8HvnUMk6dMpYvP7KRe17aR2GO\nny9eOo/yohymjyvgUFMnAkwsyaOyNJ/27jCN7QGy/MKGg0389uX9APz8/YvJ9vv46PKZfHT5zLht\nvfaMKcdsqxybb8sHpJvTHyrGTLKdn10tNq1zoMw8H770BrzxhC3JcLyTjscKPtjKpGWzE5+z4Jrj\n+04lpSTj9JcCu4wxewBE5EHgGsAr+guAm533zwBuqcDLgSeNMQ3OuU8CK4AHjr/po5dgOMKP/v4G\nm6qamDAmjwvnVXD1aZP5+7ajfO6B1wmGI8yuKKIwx8/+hg6MgYvnj+f3aw5QUZTLq3sb+H9XLWBq\naT4v76lnT207c8YX8c2rFzK2IAeAH73nNCaV5LNg8hgqim1p3HedeaxIe7n2jCm8/bTJvLa/kStP\nmTS4H2cMhDoz1+kDnP4B6G5JbhrIeBSWw+IPDG2blFFDMqJfCRz0rFcBZ8ccsxF4FzYE9E6gWETK\nEpxbGfsFInIjcCPAtGlxUtWUHqqbOrn5oQ2s2dvAosoSdtfW8afXD7Hy+T1sP9zCaVPHcuHc8Wyp\nbiYYjlBZms/nLp7D+OJcLvzhs/zoyTeYN6GYG5ZNJ8vv47KF8VPqsvw+vnbF/AG376wZ4zhrxrjB\n/8BwEEwktYOzhpuLbkl1C5RRTDKiH697Prb26peAn4vIR4DngUNAKMlzMcasBFYCLFmyJIOmDjp+\n1u9v5KG1B6hp7WZ3bRsHGzrJyfLx0/edzjsWVxKJGB5ed5A7/raDC+ZWcNcHzqAgJ/4/6+cvmcN/\nrt7Of1x1Mln+NE3cCnXaZaoHZylKhpKM6FcBUz3rU4Bq7wHGmGrgWgARKQLeZYxpFpEq4MKYc589\njvZmLJ2BMFurmynIyaKkIJtAKMLqzYf58ZNvUJSbxfSyAk6ZXMINy2Zw8fzxzKooAsDnE65bOo13\nnTmFLJ/0mRP/sfNm8vbTJjNhTBq76GCXXWay01eUFJKM6K8F5ojITKyDvw54v/cAESkHGowxEeAW\nbCYPwBPAd0Wk1Fm/zNmvAI3tAZ7cdpS/bT3Ci7vq6A4dmwd/1amTuP3aRRTnZff5WdlJOHcRSW/B\nB3X6ijLM9Cv6xpiQiNyEFXA/cLcxZquI3AasM8aswrr520XEYMM7n3XObRCRb2NvHAC3uZ26o42m\njgDX/3INXcEw588pZ29dOy/triccMUwdl8/7z57G8tnlhCIRmjqC5Gb7mFCcx7LZZQMaCXvCo05f\nUYaVpPL0jTGrgdUx277uef8I8EiCc+8m6vxHJd2hMDfeu57dNW2cPWscD609yKSSPD75lllcuWgS\nCyePGV3C3hfBDrtUp68ow4KOyB1mtlW38L2/7eDVvQ3ced3pXHN6JeGIwSeo0McjpE5fUYYTFf1h\nYldNG3f83w7+sf0ohTl+vvn2BVxzus1W9ftU7BMS1Ji+ogwnKvpDgDGG6uYu6lq72Xm0lWd21PD3\nbUcpyPbzb5fO5cPLZlBS0HdHrOLQ4/RV9BVlOFDRP04a2wN8+ZFN/GP70Z5tE8bk8uFl07npopMo\nK8pNYetOQFynr6KvKMOCiv5xsGZPPZ9/cAMN7QFufutcTqkcw5TSAuZOKNJ4/WBxnX4ml2FQlBSi\noj8IAqEI//Psbu586g2mlxXy6A3nckplSaqblRmo01eUYUVFfwAYY3ho7UH+6+ldHGrq5J2LK/n2\nO07pqWipDAHq9BVlWFG1GgD3v3qAW/+0hdOmjuU77ziFi+aPT3WT0o9IxI6q9c4WNRCC2pGrKMNJ\nmlbdSj+qmzq5ffUOlp9Uxp8/c64KfiI2PgA/XhAV74ES6gTx22n7FEUZclT0k6C1K8jXHt1MOGK4\n49pTR3cnbXcb/OQU2PNs/P1HNkFXk527dTBk8gQqipIGqOj3gTGG7/1tB+d89ymef6OWW66cz9Rx\nBaluVmppOwrNB2HH6vj7Ww7ZZUedXb7wU7j/uuQ/P9MnUFGUFKMx/T54YVcd//Psbi5fOIFPXTCb\nxdNK+z8p0wkH7PLQuvj7mx3Rb6+3ywMvw5t/dxx8EmKuTl9RhhV1+n2w8vk9VBTn8rPrF6vgu4S6\n7fLwpvhx+x6n74h+W42dCat2R3KfH+xQp68ow4iKfgxdwTCBUIQth5r555t1fHT5DHKz/KluVvrg\nOv1IEI5s7r0vFLAiD9Hwjhvbr9lGUoSSfCJQFGVQaHjHQyRiePt/vcDRli7Ki3IpzPHzgbOnp7pZ\n8dnyKETCcOp7RvZ7XacPNsQz9azoems1PbNhdtTbSc5d0T+6NbnPD3ZqsTVFGUbU6Xt4cXcdb9a0\nMX/SGA43d/Ev582kJD9NUwdf/jn880cj/71hj+hXxcT13Xg+QHsddLdGB1slK/rq9BVlWFGn7+H+\nNQcoLcjm3o8tJdvnI60yMyNh+8rKsevttdBWa930cDd0w/0w6XSYsMCGcADGToOqtb2Pc+P5WXnW\n6bsuPyt/YE4/X/tPFGW4UKfvUNPaxZPbjvLuM6eQm+XH188k4yPO09+BXyyPrrfX2/TG1iPD+72h\nAPzlJnjtt3bddfrTz4Om/dbRuzRX2eX4Bb1Ff/q50F5jb1LxMAb++WNo3GedvnbkKsqwkZToi8gK\nEdkpIrtE5Gtx9k8TkWdE5HUR2SQiVzrbZ4hIp4hscF6/GOofMFTc98oBQhHD9UunpboplkC7HQBl\nDIRD8Pq9UPeGFeFAOwTb7XGNe4e3HY17wYSjYRrX6U9fZpfVG6LHthyCvLH2KaCjPtqpO+tCu6xJ\n4Pbb6+Cpb8GalZqyqSjDTL+iLyJ+4C7gCmABcL2ILIg57D+Ah40xi4HrgP/27NttjDndeX1qiNo9\nZGw42MQ7//tFfvbUm1wwt4KBW0+AAAAgAElEQVRZFUWpbpJly6Pwu2tgzzOw7/moa26v6e2uG/YM\nbzvq3rRLtwPXdfpjnQ5uNzUTbEy/ZAoUlNk2um2edaFdHvVk8PzmSnj2e/Z9V7NdHnhJB2cpyjCT\nTEx/KbDLGLMHQEQeBK4BvDl4BhjjvC8BqoeykcPJ1/64ifr2AN+6eiHvWTIl1c2J4orpM9+F8rnR\n7a1Hex/XkKTTH2zsv+4Nu+xx+o7oFzm1h7qaose2VMGYSigsh85GO3oXYPzJUDi+d1y/dqe9OXg/\n4/BG8GWr01eUYSSZ8E4lcNCzXuVs8/JN4IMiUgWsBj7n2TfTCfs8JyLnx/sCEblRRNaJyLra2kHW\nbBkEbxxtZceRVm666CRuOHcGBTlp1K8daLPLqrWw8UEbJwcrpN66NsmEdw6uhe/PjLr2gVC/yy57\nnL4T3il0RL/TI/rNh6Ck0hFzY4U9v9QWT6uYF72BgL2JuGLvLk3EPkmo01eUYSMZ0Y9nD03M+vXA\nPcaYKcCVwL0i4gMOA9OcsM8XgftFZEzMuRhjVhpjlhhjllRUVAzsFxwHqzZU4xO4ctGkEfvOhATa\n4f++Gg11dLdCTrGNj5swnPMZu73tSFT0x81Kzuk/fZt13gdeHni7esI7MU4/pwByiqLtDXRAZ4N1\n+q6Dr90Bhc6/Z16J/Y0uwU7bJuh94wB1+ooyjCQj+lXAVM/6FI4N33wMeBjAGPMykAeUG2O6jTH1\nzvb1wG5gLmmAMYZVG6tZflI5FcVpMI/tgZdhzS9g/0t2vbvVCuWKO+CkS2HRuwGx4R13tOuUpf07\n/X0vwt7n7fua7QNvV31MTN9d+nNt+1yX3uL8lyiZYsM7YJ8S3CeC7HwbrwcIB+2NrNO5Ybg3juLJ\ndqlOX1GGjWREfy0wR0RmikgOtqN2VcwxB4BLAETkZKzo14pIhdMRjIjMAuYAw9zzmBwbDjZxoKGD\nt582OdVNsbipl10tdtndCrnFMP9t8MFHrGgWlDlOvw6yC23efGfjsU7Zy7O3Q9EEqJg/cNFvr4+6\ncdfph7vBlwU+n83UcQW7xUnX9Dr9SAiKHKeflRet1eNOiRgb3pm3wi7V6SvKsNGv6BtjQsBNwBPA\ndmyWzlYRuU1ErnYO+zfgEyKyEXgA+IgxxgBvATY52x8BPmWMaRiOHzJQVm2sJsfv4/KFE0f+y//y\nWXjzH723tR62S294Jzcmk6h4onX67bVQWAalM+32RG6/bhfs+yec+zmYvDj5omcursvPyvM4/YB1\n+WCdfmeM0x8zGQrKo5/hhney820xNYjeQLpbbDpqVzP4c2D2xdHvUxRlWEiq59IYsxrbQevd9nXP\n+23A8jjn/RH443G2ccgJRwyPbzrMRfMrRr7MQqAdXr/PjlKd89bodtfpd3tEPy+m+6NovO3IDXdb\nMR3niH7DXivqsTTtt8vKJXY078YHrHNPdsSrG88fv8AKNDgdrY7o54+FJqeP3+1nKBpvBdzFDe9k\n5Xn6BTzVObua7StvLMx8i31Vnplc+xRFGTCjckTumj311LZ2c/VpsUlIw8SLP4PXfmffu47Ym98O\nx4Z3Am02vOOlaGI0e6ewAkpn2O2JnL779DBmUjT7p2aHHfR19xXRgVaJqH/TCnj5nN4x/SyP03dD\nM+21Vthziuz+XOeG5cb3s/Ot2BvTuyRzV5N9Wsgrsa8bHrNhK0VRhoVRKfqrNlZTmOPnkpNHaJ7b\nNb+A1+6172NnlnLpcfqemH5OjOgXT7Ci31ZrxTS32Ip/fYJuElf0iybC+Pn2fc02ePFOOxCq5VD8\n81zq3rQZQtkFnph+IOrkvTH99nob1nHHAhSMc77b4/TBfo7boQtW8Lua7VODoijDzqgT/e5QmNWb\nD3PZwonkZY9AnfxAhxXXpgN23a1E2RHTtXFMR24Cpx8J2c5cN1Y+bRnseCx6npeWwzaUk50HJVOt\nC9/9dHR+277msa3bBQdesQPDsvKiTwWhmPBOd4sNHbXXRl09ROP63uwdsJ24Xqff2Wjdfl5J4rYo\nijJkjDrRf/6NOlq6Qlw9Ulk7buil7YgVOze84y2lEInY/WBF1Bi7jO3ILfI8mbiif94XrFNe9+tj\nv7v1SDQNUsRm8Ox43A6CgmhtnFgOb4S7LwfxwQVftSLfy+l7wjtgv7+jrrfou+/dZSKn3+U4fRV9\nRRkRRp3oP72jhuLcLM6bU97/wUNB/e7o++aDvacTNJ4JRyIh+76r2RnEZI51+sWeTCNX9CvPtFkv\nL/3cPlV4aa3ufY4b4nH7AuI5/XAQHvqQFel/eQImnmLfh7tte0Pd0fLOeU5IpqvJ3sQKPQPr3LTN\nojhO3zsRi5tymqfhHUUZCUad6L+6t56zZo4j25/kT2+vs2mFg6XBI/pN+6NOPxKMxu/d2Lsv24Zp\n3BIMx4R3JkTfu6IKcP6XrNPe8Pvex7cesZ24Lm5n7jmftct4or/pIdvOq34M5SfZbW44J9Qd3+l3\nuqLvuZGWz3VCSoXOZ3icfjBOTF+dvqKMCKNK9Ovautld285ZM8Yld0IoAD87A17/3eC/tH63FXOw\ncX1v56mbwePG88tm2xtBd6tdj+3I9Yq+11XPWG5j50c2RbdFwrbTt9gj+vOvglPfB6dfb511bHgn\nHILnf2gnTJlzWXR7r9CMx+m7na+th23Ixpufv+wm+Mwr0fXsArsMdvVO2WypsqNztSNXUUaEUSX6\n6/bZztOlM5MU/Y56mzffuH/wX9qwFyafboXfFf2xTs3+dlf0HadfPtc6ffcJINbp5xbZzljoLfpg\nHbW3g7StxsbuvaJfOh2uXRnN+ol1+pv/YPsgLvhq74qcvZx+97FO3y3K5m2TP6t3n4Q7BWKoM+r0\nxW8nTvF+lqIow8qoEv1X9zaSl+1jUWWSAuOWIOiOkxmTLA27oWyOrUlTu9N+5sRT7b5Yp18+106O\n4o5yje3Ihajb94Z3oPeIV4jeSIoTFJMrGn+s6G/7s03RnHdF7+29nH7g2Jh+j+j30U+S5Y3pOzen\n4oke0VenrygjwegS/X31LJ5aSk5Wkj+700mrdHPRB0qg3Ypv2Szr7g844Y4e0XcyeFoPWxF3RdON\n+8c6fbCinzc2Krwu7uAnlx7RT1BmorDi2PBOW40V/di6+/06faffoi/Rd51+rOi7Uyyq01eUEWHU\niH5rV5Bt1S3Jh3Yg6vTj5cAD7HoKfnmJnQHKzcP34s5qNW62FX33JjIpjtMvnhQdxerG/eOJfsVc\nG/uPJSu/dwdpz2jcBKmp8cI7sRk4PZ8dG9N3RD+n0BZfc51+QRJOP9QVDUO54w5AY/qKMkKk0awh\nw8v6/Y1ETB/x/L3/tJN9TDsnus0dQBUvvHN0Gzx8g41dP3s7vHwXfGFj77o2rgMum907dbN8rnXL\n7R6nXzwxWmvHdb+xHbkAl98encjES3Z+79IOLYdtzDyeiIMN73Q19Q7XdNQdGzaC3k4/1B0dkSti\nHbo7Q1bSTr/T/n7vtVKnrygjwqhx+lurrXCfNjWBo3zy/8Ez/9l7WyKn39kI97/POt1PvQgfetR2\n+O55rvdxbrrmuFnRzltwKlGWRW8qrUes6Pc4/T7COzkF8V1xdl6M0z9iQ0G+BKOO3ZuB6/YD7bZP\nIK7Td0W/y95w3HWIxuKzC6LpmfGIdfrZeb1/h8b0FWVEGDWiv6+unYriXIpyEzzctNXa0gdeOhM4\n/b3PQ/MBeOcv7PSAM94CuSWwK6Zccv0eK7y5xVHRzx9nXXlhmXXWkbCd7Lx4UtTptxyyYROvuPZH\ndkHvka6xA7NiiRV9d9lXeCfc3Tu8A1GH3pfLh2OdflZ+b9HPPWZCNUVRhoFRI/r7GzqYUVYQf6cx\nVvSCMSNae5x+TEeu68TdDll/Fsy6wNa1MZ6ZJFuqomLvLsc4lT0Lymw4pie1cmJUQFuq7Y1iIBOZ\nZ8Vx+oni+RAdKdsj+k6oqS+nH+zq3ZELUeHuK54P8Z2+6+5ziu01VBRl2Bk1on+gvoNp4xKEHwJt\nVsxiyxi44ZdAm3XkLi3VVvgKPP0DJ73VOnTvRCWdTdG4dfFEm6tf4op+uRX9I5vtetkc+7QATt2d\nOKGdvsgu6J2n7/YTJMIVdzeDp8fpxxFv1+m7I4W9mUM9Tr+fuY19PnvN3OydrLzotdFOXEUZMUaF\n6HcFwxxp6WJ6Iqfvutxge+/t3mkIvSGe1sO2vIHXiZ90iV16Qzze8gI+v81/n3WhXS8os4Oz9r9o\nbwZTzuo9aUq8Tty+yM6LPqm4k473JfoJnX480Xecvdu34Y8T0+8vvOO2MdTlEX3nXO3EVZQRY1SI\n/oEGK4b9in4gVvQ95Y+9nbkt1dHqlS4lU6Di5Dii73Gx77sXzvm0fV9Ybjt/9z4HlWfYDlp/djQM\nMhinHwnaUgpuWKqvkEtOoT0nNqYf7xzX6buzeg0mpg/RtNJgp+3XyHOcvnbiKsqIkZToi8gKEdkp\nIrtE5Gtx9k8TkWdE5HUR2SQiV3r23eKct1NELh/KxifL/nor+tPGJRB9d5BUqKt3GKezMSpI3TGi\nPybOSNeZb4EDa2xc35i+C4m5oaHq12H6udHtrtuPNxq3L7I8ZQ7cm1d/Nw7vAK32OlviISfONTrG\n6XvCO8nG9CGaYaROX1FSRr+iLyJ+4C7gCmABcL2IxM5n9x/YCdMXA9cB/+2cu8BZXwisAP7b+bwR\nZX+9FcEZZQli+t5BSm6IxBgb03fLELuducY44Z04naRjJkVrywTabCGxhKLvEcnpnumF3eMH7PQ9\nZQ7c2HtfKZTQuxRD7CQoXnqcviP6cZ1+PzF9sE4/5Eyikp2vMX1FSQHJOP2lwC5jzB5jTAB4ELgm\n5hgDuAHpEsBJb+Ea4EFjTLcxZi+wy/m8EeVAQwfFeVmMLUgwCbp3QpOAJy4e7vaIviN4nY3WqcaG\nd8CmY4INC7n9AYkEzR0EJT6Y6rkkburi8Yh+d5Ki7x2V6867Gw9/H05/oDH9oCem794w1OkryoiR\njOhXAgc961XONi/fBD4oIlXAauBzAzgXEblRRNaJyLra2j6m8Bsk++s7mF5WgCRKgfSKvtuZ68bz\nS6fbpety3XTNeOEdN2TT0RB9MkgkaK5ITlzU+xg3vDPgjlyv03d+QzKi74Z3OuoSh2h8Piv08Zz+\nhIV2LML4k/tvY1Z+745cnx8u+Tosenf/5yqKMiQkI/rxlNLErF8P3GOMmQJcCdwrIr4kz8UYs9IY\ns8QYs6SiIokwwQDZX9/O9ETpmtB7knJXMN3O0Fin31PIrB+n35/ou07fG9qBwTv9njx4b3inn36B\n4kn2twc6jp0E5ZjPz4ufvVMxD770hu3I7g83ph/sjA7WOv/f7OxfiqKMCMmIfhUw1bM+hWj4xuVj\nwMMAxpiXgTygPMlzh5VQOEJVYyfTEmXuQPzwjpujP9Z1+o6Iu8XQ4sX0ezl9J7yTKDOlsMK63KU3\n9t4+2I7cuE6/n8+oPMMODDu0vu/wDsQ4/ZzEx/VFL6efP7jPUBTluEhG9NcCc0RkpojkYDtmV8Uc\ncwC4BEBETsaKfq1z3HUikisiM4E5wKtD1fhkONzcRShiEo/GBSt47sCoYIzTL57ouFxX9A8DEj8H\nfiBOX8S63HEze28fko7cJMM7U86yyzefsNUu+xL9RE5/QG2M4/QVRRlR+hV9Y0wIuAl4AtiOzdLZ\nKiK3icjVzmH/BnxCRDYCDwAfMZat2CeAbcDfgM8aY8LHfsvwEU3X7Cu8Uw+lTpkE1+m7Mf38UivE\nPeGdaiuO/jidwm42Skdj/6KfiNwRFP2CcVA+D7Y/btf7FP3c43f62fnRrCZ1+oqSEpIqeGKMWY3t\noPVu+7rn/TZgeex5zr7/BP4z3r6RoLrZ1qOpHJtAZNy6O5MX25IIwZjwTn6pjbP3dOQejt+JC1YM\nc4rtDcOtEz9Q0R9sR663tk2gzbrxeDemWKadDa85cwAXximr3PP5edG+gsE6/az86BPUQIrJKYoy\nZGT8iNyGdlt7vqwogTvtbrXlgt2CaN6O3Kx8Z+TomKjTb6mOFk2LR0FpNHsnd0zi0saJOO6UzQ77\nG/pz+S5TPfMH9Of0e94PMjSTnRedCyBbnb6ipIJRIfq5WT4KchKIr5u543bYekXf7Zj1Ov3W6sTz\nzoKN63c6HbmDyT8vn2s7Tb3195OhR/S7HNFPsiN4WrKi7xH64+nIjfd5iqKMGBlfz7aurZuywpz+\nc/RdkXXDO52N0Rh93hibteMWMksU3gF7o+hosPXwB1NTZsqZ8O/VyYVmvPRy+m3JO/1xs5yKnwlm\nzXLxOv3j6cjtea9OX1FSwahw+uMShXYgKvrFE63Ddp1+R0NU9HOd8I47MCtejr5Lj9Pvo+5OfwxU\n8CEmpj+A8I4ITD3btruv71WnrygZQcY7/Yb2AGWFMc40HIpO2uGtI59d0NvpV8yz7/NKbHjn6Fa7\nXnZS4i90nX5WfnRg10jQU69+gE4f7HiBxr19HzPUTl9FX1FSQsY7/fq2AGWFHme67m74yQI7sxRE\nY/oF5VYovSmbbkw/r8SK6Z5nrZhPXpz4C/PHWZff2TDyhcSy86Mx/YF0BI+fb2v990Uvp38c2Tsu\nmqevKCkh40W/oT3AOK/o1+6EtqPw+BeddM06yC60JYWzC+zgrEjECe94OnIB3vibTXHsK7yRXwo4\nlThHupBYdv7gnH4yuEIv/oFnJLn0cvoa01eUVJDRot8RCNEZDPeO6bvVL3f+FZ69A/a/FM1Pzym0\nLrmryQ4gcrNZvBOWzziv7y/1TqGYCtEfaEw/WVynfzz59dmeUdHq9BUlJWR0TL++zcnR9zr9riYY\nv9CKznN32M5bt/6NG97pqLfrbgEy1+kDzDi/7y/N94r+CId33JmphkX0nWvoH2QnLsSEiFT0FSUV\nZLTouwOzxnk7cruarRt/zz021FN5RjR9MLvAxvh7pg50ngBcx56VD5PP6PtLC0qj71Ph9ANtNsST\nbJ5+sgyJ09fsHUVJNaNC9Mtiwztls62Ljy0lnFMATR2eScJjwjtTl/afruh1+qnoyHWfUoYrpj/Y\nzB3oLfSap68oKSGjY/r17fHCO82JxTi70LpkN6PHvSm4YZr+4vmQ+pi+e8Matpj+cYR31OkrSsrJ\naKdf39YN0Dt7p6spcaw9p8CGR9odt+yGd8ZOg7f9CBZe2/+X5o6xo3EjoZEX/aw8j+gPU3hnqJy+\nir6ipISMdvoN7QFy/D6Kcp17WyhgnXxC0Xc6cttrrXj3pCkKnPXx3i4+ESKe8g0jHd4pgEjQvh+u\n8M5QOH1/rh1MpijKiJPRf3n1To5+T90dt8Z9X+GdcLfN409mou9E5HsGdY0k3jTIdHb66vIVJWVk\ntOgfMzCrZwrDBGKc4+SRNx9MPEl4MhSMs4OYhtpt94c3D37IRT+393Iw9GRJqegrSqrIaNGvbw/0\nztzpmc0qkdN3RLNx//E7/fyxNtQzkngddDoOzvL5wZetTl9RUkjGd+TO9M6N29mf03eEsr8yw/0x\nY3lqZobq5fSHK2XzOGL6YNuo6ZqKkjKSEn0RWQHcCfiBXxlj7ojZ/xPgIme1ABhvjBnr7AsDm519\nB4wxVzNC2PCOd2CWI/qJYvpeoexrQpH+WPbZwZ97PGSnudMH20Z1+oqSMvoVfRHxA3cBlwJVwFoR\nWeXMiwuAMeZmz/GfA7xlKDuNMacPXZOToysYpiMQjgnvuE6/n/AOHF94J1V4HfRwxfSPpyMXrOCr\n6CtKykgmpr8U2GWM2WOMCQAPAtf0cfz1wAND0bjjob6nBEOcYmv9hXfg+DpyU4VbudKXfXyplfHw\nD0HKJtgbk3bkKkrKSEb0K4GDnvUqZ9sxiMh0YCbwtGdznoisE5FXROQdCc670TlmXW1tbZJN75uG\nuMXWmq3LTCQ6vZz+ccT0U4Xr9Icja2ionP64WVA68/jboyjKoEgmph8vBcUkOPY64BFjTNizbZox\nplpEZgFPi8hmY8zuXh9mzEpgJcCSJUsSffaAaOq0oj+2IHY0bh+580MV008VPaI/xKEdGJoyDADv\nu4/4/6UURRkJknH6VcBUz/oUoDrBsdcRE9oxxlQ7yz3As/SO9w8bHQF73ynI8Uz40dXc9yjZEz28\n44p+7jCK/vE6fZ9fR+MqSgpJ5q9vLTBHRGaKSA5W2FfFHiQi84BS4GXPtlIRyXXelwPLgW2x5w4H\nnY7o53tFv7Mfp3+id+RmjUB4JxWpqIqiDBn9hneMMSERuQl4ApuyebcxZquI3AasM8a4N4DrgQeN\nMd7wzMnA/4pIBHuDucOb9TOcxHf6TVA0IfFJrljmFJ+Y4jacMf2cQjvZzElvHfrPVhRlxEgqT98Y\nsxpYHbPt6zHr34xz3kvAouNo36DpCIQAKMj2/MSuZiifl/gkn9+GL05Elw/DG9MXgSt/MPSfqyjK\niJKxwdVBhXfA1t854UV/hGv+KIpywpCxot8RDJPlE3KynJ8YiUB3S/+zWeUUnZiduDC8MX1FUTKC\njK290xkI93b5gVYwkf6d/hkftrnkJyLDGd5RFCUjyFjR7wiEenfidvZTgsHlgq8MX6OGm6w8K/hF\n41PdEkVR0pQMFv0wBTkxnbgw8hObjCQ+H9z4HIyZnOqWKIqSpmSs6HcGwuRnx6RrQv8x/ROd8pNS\n3QJFUdKYzO3IDYSPHY0LIz9vraIoShqRuaIfDB+brgmZHd5RFEXph4wV/c7YjtzaHXbWpxOxkJqi\nKMoQkbGi36sj1xjY/hjMvCA6+bmiKMooJGNFv1ee/tEt0LQfTn57ahulKIqSYjJW9DsCYQrc7J3t\nj4H4YN6VqW2UoihKislI0Y9EDJ3BEHM7N0B3G2x/HKYtgyKN5yuKMrrJSNHvCoVZLLt479ZPw08W\nQM1WmH9VqpulKIqScjJS9DsCYYqlw66UnQTFk2BBX3O5K4qijA4yckRuZyBMFs40vVf+ACrPTG2D\nFEVR0oSMdfrZruj7slPbGEVRlDQiQ0U/FHX6vox8mFEURRkUSYm+iKwQkZ0isktEvhZn/09EZIPz\nekNEmjz7bhCRN53XDUPZ+ETY8I6dLhG/On1FURSXfm2wiPiBu4BLgSpgrYis8k5wboy52XP854DF\nzvtxwDeAJYAB1jvnNg7pr4ihIxAmi4hdUaevKIrSQzJOfymwyxizxxgTAB4E+kqFuR54wHl/OfCk\nMabBEfongRXH0+Bk6AiGyRInvKNOX1EUpYdkRL8SOOhZr3K2HYOITAdmAk8P5FwRuVFE1onIutra\n2mTa3SedgRDZbnhHO3IVRVF6SEb0Jc42k+DY64BHjDHhgZxrjFlpjFlijFlSUXH8o2Y7vCmb6vQV\nRVF6SEb0q4CpnvUpQHWCY68jGtoZ6LlDRi/R9/n7PlhRFGUUkYzorwXmiMhMEcnBCvuq2INEZB5Q\nCrzs2fwEcJmIlIpIKXCZs21Y6QqGyRbN01cURYml39QWY0xIRG7CirUfuNsYs1VEbgPWGWPcG8D1\nwIPGGOM5t0FEvo29cQDcZoxpGNqfcCwdgTAlPid7R8M7iqIoPSSVz2iMWQ2sjtn29Zj1byY4927g\n7kG2b1B0BMJM8Bvbe6Apm4qiKD1k5IjczkCIfH/ECr7E60tWFEUZnWSkDe4IhK3oo6EdRVEULxkp\n+p3BMLm+CJiM/HmKoiiDJiNVsSPgiH5m/jxFUZRBk5GqGBV9De8oiqJ4ydiO3FyJaLqmoihKDBkp\n+h2BMDm+sKZrKoqixJCRot8ZCJMjYXX6iqIoMWSc6Btj6Ag6oq9OX1EUpRcZJ/qBcIRwxJAtRuvu\nKIqixJBxot8ZsIXWsgiBX52+oiiKl4wT/Q5H9LMJq9NXFEWJIWNFPwvtyFUURYkl40Q/ELIllf2E\ntCNXURQlhowT/XDElvP3GRV9RVGUWDJO9IMR6/R9RsM7iqIosWSc6Pc4/Yg6fUVRlFgyTvSDYdfp\nh9TpK4qixJCU6IvIChHZKSK7RORrCY55r4hsE5GtInK/Z3tYRDY4r2MmVB9qXKcvJqQpm4qiKDH0\nG/8QET9wF3ApUAWsFZFVxphtnmPmALcAy40xjSIy3vMRncaY04e43QkJecM76vQVRVF6kYzTXwrs\nMsbsMcYEgAeBa2KO+QRwlzGmEcAYUzO0zUyeUNjr9P2paoaiKEpakozoVwIHPetVzjYvc4G5IvKi\niLwiIis8+/JEZJ2z/R3H2d5+CTvZOxLR8I6iKEosyaS3SJxtJs7nzAEuBKYA/xSRU4wxTcA0Y0y1\niMwCnhaRzcaY3b2+QORG4EaAadOmDfAn9CboOn0N7yiKohxDMk6/CpjqWZ8CVMc55i/GmKAxZi+w\nE3sTwBhT7Sz3AM8Ci2O/wBiz0hizxBizpKKiYsA/wktPR24kqCmbiqIoMSQj+muBOSIyU0RygOuA\n2CycPwMXAYhIOTbcs0dESkUk17N9ObCNYcRN2VSnryiKciz9WmFjTEhEbgKeAPzA3caYrSJyG7DO\nGLPK2XeZiGwDwsCXjTH1InIu8L8iEsHeYO7wZv0MB9bpG8fpq+griqJ4SSr+YYxZDayO2fZ1z3sD\nfNF5eY95CVh0/M1MnmDE4HO7HDS8oyiK0ouMG5EbDkfIJmRXdBIVRVGUXmSc6IcixtbSBw3vKIqi\nxJDZoq8duYqiKL3IPNEPR+xUiaAxfUVRlBgyT/Qjxk6KDur0FUVRYsg80Q8bssXm6qvTVxRF6U3m\niX7EkOd3RV+dvqIoipfME/1whDyfI/qasqkoitKLzBP9iCHXpymbiqIo8chA0Y+Q2+P0VfQVRVG8\nZJzohyOGXHHLMKjoK4qieMk40Q+GveEdnTlLURTFS8aJfjhiNLyjKIqSgIwT/WA4Qo5oR66iKEo8\nMk70ezt9TdlUFEXxknGiHwwbdfqKoigJyDjRD0ci5GhMX1EUJS4ZJ/qhiCFbtMqmoihKPDJP9MOG\nXC24piiKEpekRF9EVomR2UUAAAaGSURBVIjIThHZJSJfS3DMe0Vkm4hsFZH7PdtvEJE3ndcNQ9Xw\nRIQinnr6Gt5RFEXpRb9WWET8wF3ApUAVsFZEVhljtnmOmQPcAiw3xjSKyHhn+zjgG8ASwADrnXMb\nh/6nWEIRQ47W3lEURYlLMk5/KbDLGLPHGBMAHgSuiTnmE8BdrpgbY2qc7ZcDTxpjGpx9TwIrhqbp\n8QlHDFloR66iKEo8kgl6VwIHPetVwNkxx8wFEJEXAT/wTWPM3xKcWxn7BSJyI3Cjs9omIjuTan18\nyh+Dug8BfKvsOD5myCkH6lLdiDika7sgfdum7RoY6douSN+2DaZd05M5KBnRlzjbTJzPmQNcCEwB\n/ikipyR5LsaYlcDKJNrSLyKyzhizZCg+ayjRdg2cdG2btmtgpGu7IH3bNpztSia8UwVM9axPAarj\nHPMXY0zQGLMX2Im9CSRzrqIoijJCJCP6a4E5IjJTRHKA64BVMcf8GbgIQETKseGePcATwGUiUioi\npcBlzjZFURQlBfQb3jHGhETkJqxY+4G7jTFbReQ2YJ0xZhVRcd8GhIEvG2PqAUTk29gbB8BtxpiG\n4fghHoYkTDQMaLsGTrq2Tds1MNK1XZC+bRu2dokxx4TYFUVRlAwl40bkKoqiKIlR0VcURRlFZIzo\nJ1MqYoTaMVVEnhGR7U5Jis8728eJyJNOOYonnY7tVLTPLyKvi8jjzvpMEVnjtOshp7M+Fe0aKyKP\niMgO59otS4drJiI3O/+OW0TkARHJS9U1E5G7RaRGRLZ4tsW9RmL5mfP3sElEzhjhdv3A+bfcJCJ/\nEpGxnn23OO3aKSKXj2S7PPu+JCLGSTwZ0evVV9tE5HPOddkqIt/3bB+6a2aMOeFf2A7m3cAsIAfY\nCCxIUVsmAWc474uBN4AFwPeBrznbvwZ8L0Xt+yJwP/C4s/4wcJ3z/hfAp1PUrt8CH3fe5wBjU33N\nsAMJ9wL5nmv1kVRdM+AtwBnAFs+2uNcIuBL4P+xYmXOANSPcrsuALOf99zztWuD8feYCM52/W/9I\ntcvZPhWbfLIfKB/p69XHNbsI+AeQ66yPH45rNuz/UUfiBSwDnvCs3wLckup2OW35C7Zu0U5gkrNt\nErAzBW2ZAjwFXAw87vwHr/P8cfa6jiPYrjGOuErM9pReM6IjysdhM90ex5YWSdk1A2bECEXcawT8\nL3B9vONGol0x+94J/N553+tv0xHfZSPZLuAR4DRgn0f0R/R6Jfi3fBh4a5zjhvSaZUp4J6lyDyON\niMwAFgNrgAnGmMMAznJ8Cpr0U+Ar4BYnogxoMsaEnPVUXbdZQC3wGyf09CsRKSTF18wYcwj4IXAA\nOAw0A+tJj2vmkugapdPfxL9gXTSkuF0icjVwyBizMWZXOlyvucD5TujwORE5azjalimin1S5h5FE\nRIqAPwJfMMa0pLItTnuuAmqMMeu9m+McmorrloV91P0fY8xioB0bqkgpTnz8Guwj9WSgELgizqHp\nmPecFv+2InIrEAJ+726Kc9iItEtECoBbga/H2x1n20hfryygFBte+jLwsIgIQ9y2TBH9tCr3ICLZ\nWMH/vTHmUWfzURGZ5OyfBNQkOn+YWA5cLSL7sJVSL8Y6/7Ei4g7SS9V1qwKqjDFrnPVHsDeBVF+z\ntwJ7jTG1xpgg8ChwLulxzVwSXaOU/02InT/jKuADxolLpLhds7E38I3O38EU4DURmZjidrlUAY8a\ny6vYJ/LyoW5bpoh+MqUiRgTnzvxrYLsx5seeXasAdxKZG7Cx/hHDGHOLMWaKMWYG9vo8bYz5APAM\n8O5Utctp2xHgoIjMczZdAmwjxdcMG9Y5R0QKnH9Xt10pv2YeEl2jVcCHnayUc4BmNww0EojICuCr\nwNXGmI6Y9l4nIrkiMhNbo+vVkWiTMWazMWa8MWaG83dQhU26OEKKr5fDn7FmDBGZi01oqGOor9lw\ndlSM5Avb+/4Gtmf71hS24zzso9cmYIPzuhIbP38KeNNZjkthGy8kmr0zy/kPtAv4A07mQAradDqw\nzrluf8Y+5qb8mgHfAnYAW4B7sRkUKblmwAPYvoUgVrA+lugaYUMCdzl/D5uBJSPcrl3YOLT7N/AL\nz/G3Ou3aCVwxku2K2b+PaEfuiF2vPq5ZDnCf83/tNeDi4bhmWoZBURRlFJEp4R1FURQlCVT0FUVR\nRhEq+oqiKKMIFX1FUZRRhIq+oijKKEJFX1EUZRShoq8oijKK+P8BiJ6yZ+qyewAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f41c826d6d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs = 160\n",
    "\n",
    "a, b = [], []\n",
    "for epoch in range(epochs):\n",
    "    if epoch == 60:\n",
    "        trainer.set_learning_rate(0.02)\n",
    "    if epoch == 120:\n",
    "        trainer.set_learning_rate(0.004)\n",
    "    if epoch == 140:\n",
    "        trainer.set_learning_rate(0.0008)\n",
    "    train_loss = 0.\n",
    "    train_acc = 0.\n",
    "    start = time()\n",
    "    for data, label in train_data:\n",
    "        data = augment(data, aug_train).as_in_context(ctx)\n",
    "        label = label.as_in_context(ctx)\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            l = loss(output, label)\n",
    "        l.backward()\n",
    "        trainer.step(batch_size)\n",
    "        train_loss = train_loss + nd.mean(l).asscalar()\n",
    "        train_acc = train_acc + accuracy(output, label)\n",
    "    test_acc = evaluate_accuracy(test_data, net, ctx)\n",
    "    \n",
    "    if epoch%10 == 0:\n",
    "        print(epoch, 'loss:%.4f tracc:%.4f teacc:%.4f time:%.3f'%(\n",
    "            train_loss/len(train_data), train_acc/len(train_data), test_acc, time()-start)) \n",
    "    a.append(train_acc/len(train_data))\n",
    "    b.append(test_acc)\n",
    "\n",
    "print('tracc:%f teacc:%f'%(train_acc/len(train_data), test_acc))\n",
    "plt.plot(np.arange(epochs), a, np.arange(epochs), b)\n",
    "plt.ylim(0.6,1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
